# -*- coding:utf-8 -*-
from database.db_business import DbBusiness
from common.my_time import MyTime
from common.my_baidu import MyBaidu
from common.attr_v3 import AttrV3
from bs4 import BeautifulSoup
import glob
import jieba
import re
import threading
import hashlib
import urllib.parse
import time
import os
from common.market_data import MarketData


'''
网络数据
'''

class BaiduGangkou:
    def __init__(self):
        self.attr_v3 = AttrV3()
        self.business = DbBusiness()
        self.market = MarketData()

    def check_site(self, site_name):
        site_lists = ["站长之家","中关村在线","新浪","搜狐","东方财富网","同花顺","网易","金融界","证券之星","每日经济新闻","中国财经信息网","凤凰","和讯","中国经济网","格隆汇","中国网","人民资讯","中证网","第一财经","证券时报","华夏时报","中华网","经济观察报","人民网","中国民航网"]
        for d in site_lists:
            if re.search(d, site_name):
                return True
        print(site_name)
        return False
    
    def get_data(self, word):
        datas = ["丹东港","大连港","营口港","锦州港","秦皇岛港","唐山港","天津港","黄骅港","龙口港","烟台港","威海港","石岛港","张家埠港","青岛港","日照港","岚山港","连云港港","南通港","张家港港","镇江港","南京港","上海港","乍浦港","宁波港","舟山港","石浦港","海门港","温州港","赛岐港","福州港","湄州湾港","泉州港","厦门港","东山港","汕头港","西堤港","汕尾港","深圳港","广州港","中山港","珠海港","江门港","阳江港","水东港","湛江港","霞海港","海安港","北海港","防城港","海口港","三亚港","八所港","洋浦港","马村港","高雄港","花莲港","台中港","基隆港","香港港","澳门港","海县港","嘉定港","宝山港","松江港","金山港","青浦港","奉贤港","南汇港","川沙港","崇明港","内河港","南通港","张家港港","大中港","建湖港","昆山港","兴化港","江阴港","高港港","盐城港","常熟港","南京地方港","泰兴港","东台港","滨海港","梧桐港","冉里山港","硖石港","萧山港","雉城港","魏塘港","小浦港","周浦港","泗安港","湖州港","平湖港","李家巷港","马鞍山港","芜湖港","铜陵港","安庆港","荻港港","池州港","散兵港","合肥港","淮南港","蚌埠港","九江港","南昌港","赣州港","武穴港","田镇港","黄石港","巴河港","兰溪港","鄂州港","武汉港","沙市港","枝城港","宜昌港","襄樊港","利河口港","城陵矶港","岳阳港","常德港","沅江港","津市港","株州港","湘潭港","茅草街港","衡阳港","万县港","涪陵港","重庆港","北碚港","节港","泸州港","合川港","宜宾港","杭州港","苏州港","常州港","邳州港","淮阴港","宜兴港","吴江港","铜山港","淮安港","徐州港","三埠港","容奇港","东莞港","肇庆港","佛山港","石龙港","市桥港","太平港","梧州港","南宁港","柳州港","贵港港","虎门港","哈尔滨港","佳木斯港","沙河子港","富锦港","同江港","黑河港","鳌江港","宝山码头港","八所港","北海港","北津港","北仑港","博贺港","防城港","福州港","广州港","海安港","海口港","海门港","黄岛港","黄埔港","锦州港","九州港","岚山港","连云港","龙口港","绿华山港","罗泾港","旅顺港","南堡港","南门港","宁波港","盘石港","澎湖港","蓬菜港","平潭港","普陀港","青岛港","清澜港","秦皇岛港","泉州港","瑞安港","乳山口港","赛岐港","三都港","三亚港","上海港","山海关港","汕头港","汕尾港","蛇口港","嵊泗港","沈家门港","射阳港","石岛港","石臼港","石浦港","水东港","塘沽港","天津港","天津新港","王滩港","威海港","温州港","厦门港","秀屿港","洋浦港","烟台港","燕尾港","营口港","营口新港","漳港","张家埠港","章湾港","湛江港","乍浦港","镇海港","中山港","庄河港","深圳港","陈家港","陈山码头港","大连港","大连新港","丹东港","大清河港","刁口镇港","定海港","东山港","盐田港","顺德港","佛山港","江门港","芜湖港","马鞍山港","安庆港","铜陵港","南京港","常州港","江阴港","镇江港","扬州港","泰州港","张家港","常熟港","南通港","无锡港","重庆港","合肥港","长沙港","九江港","武汉港","南昌港","珠海港","太仓港","漳州港","京唐港","莱州港","龙眼港","妈湾港","钦州港","日照港","台州港","阳江港","舟山港","大铲湾港","虎门港","南沙港","安庆港"]
        end_date = MyTime.forward_relative_date(3)
        check_title = [word]
        for k in datas:
            next_page = True
            words = [k, word]
            for i in range(1,3):
                #time.sleep(1)
                if next_page == False:
                    break
                url = MyBaidu.get_url_order_by_time(words, i)
                try:
                    proxy = self.business.query_proxy()
                    print("proxy:" + proxy)
                    res = MyBaidu.get_baidu_data_by_proxy(url, proxy)
                    for r in res:
                        data_date = MyBaidu.calc_date(r["date"])
                        if data_date < end_date:
                            next_page = False
                            break
                        #if self.check_site(r["site"]) == False:
                        #    continue
                        if MyBaidu.check_title(r["title"], [[k], check_title]) == False:
                            continue
                        data_title = r["title"]
                        data_site = r["site"]
                        data_url = r["url"]
                        print(r["date"] + data_date)
                        self.market.add_market_data(data_date, 2303, data_title, data_url, data_site)
                except Exception as e:
                    i = i - 1
                    pass

def gangkou_api():
    d = BaiduGangkou()
    d.get_data("吞吐量")
    d.get_data("货物")


if __name__ == '__main__':
    gangkou_api()
